{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0321 13:30:38.625199 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer\n",
      "W0321 13:30:38.626076 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.iter\n",
      "W0321 13:30:38.627406 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.beta_1\n",
      "W0321 13:30:38.627910 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.beta_2\n",
      "W0321 13:30:38.629245 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.decay\n",
      "W0321 13:30:38.632287 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.learning_rate\n",
      "W0321 13:30:38.641271 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-0.kernel\n",
      "W0321 13:30:38.643591 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-0.bias\n",
      "W0321 13:30:38.644773 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-1.gamma\n",
      "W0321 13:30:38.646014 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-1.beta\n",
      "W0321 13:30:38.646898 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-2.kernel\n",
      "W0321 13:30:38.647903 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-2.bias\n",
      "W0321 13:30:38.664048 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-3.gamma\n",
      "W0321 13:30:38.677104 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-3.beta\n",
      "W0321 13:30:38.677987 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-4.kernel\n",
      "W0321 13:30:38.678569 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-4.bias\n",
      "W0321 13:30:38.679900 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-5.gamma\n",
      "W0321 13:30:38.685153 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-5.beta\n",
      "W0321 13:30:38.686394 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-6.kernel\n",
      "W0321 13:30:38.688503 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-6.bias\n",
      "W0321 13:30:38.691724 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-7.gamma\n",
      "W0321 13:30:38.694571 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-7.beta\n",
      "W0321 13:30:38.696645 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-8.kernel\n",
      "W0321 13:30:38.703249 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-8.bias\n",
      "W0321 13:30:38.710791 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-9.kernel\n",
      "W0321 13:30:38.711881 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-9.bias\n",
      "W0321 13:30:38.712974 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-10.kernel\n",
      "W0321 13:30:38.713840 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-10.bias\n",
      "W0321 13:30:38.714511 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-11.kernel\n",
      "W0321 13:30:38.720114 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-11.bias\n",
      "W0321 13:30:38.722012 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-0.kernel\n",
      "W0321 13:30:38.724092 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-0.bias\n",
      "W0321 13:30:38.729871 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-1.gamma\n",
      "W0321 13:30:38.730692 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-1.beta\n",
      "W0321 13:30:38.731453 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-2.kernel\n",
      "W0321 13:30:38.733532 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-2.bias\n",
      "W0321 13:30:38.734930 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-3.gamma\n",
      "W0321 13:30:38.735954 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-3.beta\n",
      "W0321 13:30:38.736789 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-4.kernel\n",
      "W0321 13:30:38.737963 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-4.bias\n",
      "W0321 13:30:38.742634 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-5.gamma\n",
      "W0321 13:30:38.744631 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-5.beta\n",
      "W0321 13:30:38.746654 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-6.kernel\n",
      "W0321 13:30:38.752413 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-6.bias\n",
      "W0321 13:30:38.754378 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-7.gamma\n",
      "W0321 13:30:38.757546 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-7.beta\n",
      "W0321 13:30:38.758607 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-8.kernel\n",
      "W0321 13:30:38.759586 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-8.bias\n",
      "W0321 13:30:38.760442 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-9.kernel\n",
      "W0321 13:30:38.761178 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-9.bias\n",
      "W0321 13:30:38.761847 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-10.kernel\n",
      "W0321 13:30:38.762884 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-10.bias\n",
      "W0321 13:30:38.763983 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-11.kernel\n",
      "W0321 13:30:38.767845 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-11.bias\n",
      "W0321 13:30:38.768987 140735542186880 util.py:152] A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/alpha/guide/checkpoints#loading_mechanics for details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0321 13:30:38.770684 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer\n",
      "W0321 13:30:38.803921 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.iter\n",
      "W0321 13:30:38.810781 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.beta_1\n",
      "W0321 13:30:38.811746 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.beta_2\n",
      "W0321 13:30:38.812452 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.decay\n",
      "W0321 13:30:38.813846 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.learning_rate\n",
      "W0321 13:30:38.816632 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-0.kernel\n",
      "W0321 13:30:38.818003 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-0.bias\n",
      "W0321 13:30:38.819238 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-1.gamma\n",
      "W0321 13:30:38.820166 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-1.beta\n",
      "W0321 13:30:38.820755 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-2.kernel\n",
      "W0321 13:30:38.823518 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-2.bias\n",
      "W0321 13:30:38.824273 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-3.gamma\n",
      "W0321 13:30:38.825911 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-3.beta\n",
      "W0321 13:30:38.827776 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-4.kernel\n",
      "W0321 13:30:38.828976 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-4.bias\n",
      "W0321 13:30:38.830085 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-5.gamma\n",
      "W0321 13:30:38.831302 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-5.beta\n",
      "W0321 13:30:38.833347 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-6.kernel\n",
      "W0321 13:30:38.834519 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-6.bias\n",
      "W0321 13:30:38.835659 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-7.kernel\n",
      "W0321 13:30:38.836633 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-7.bias\n",
      "W0321 13:30:38.838973 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-8.kernel\n",
      "W0321 13:30:38.840754 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-8.bias\n",
      "W0321 13:30:38.842096 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-0.kernel\n",
      "W0321 13:30:38.843008 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-0.bias\n",
      "W0321 13:30:38.844038 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-1.gamma\n",
      "W0321 13:30:38.845168 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-1.beta\n",
      "W0321 13:30:38.846125 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-2.kernel\n",
      "W0321 13:30:38.847459 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-2.bias\n",
      "W0321 13:30:38.848613 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-3.gamma\n",
      "W0321 13:30:38.849587 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-3.beta\n",
      "W0321 13:30:38.851434 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-4.kernel\n",
      "W0321 13:30:38.852485 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-4.bias\n",
      "W0321 13:30:38.853564 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-5.gamma\n",
      "W0321 13:30:38.855915 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-5.beta\n",
      "W0321 13:30:38.857151 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-6.kernel\n",
      "W0321 13:30:38.857725 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-6.bias\n",
      "W0321 13:30:38.858959 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-7.kernel\n",
      "W0321 13:30:38.860094 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-7.bias\n",
      "W0321 13:30:38.862030 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-8.kernel\n",
      "W0321 13:30:38.863651 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-8.bias\n",
      "W0321 13:30:38.866490 140735542186880 util.py:152] A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/alpha/guide/checkpoints#loading_mechanics for details.\n",
      "/anaconda3/lib/python3.7/site-packages/distributed/config.py:20: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  defaults = yaml.load(f)\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "from io import BytesIO\n",
    "from tensorflow.python.lib.io import file_io\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "\n",
    "from tensorflow.python.keras.layers import Input, Dense, Convolution2D,MaxPool2D,Flatten,BatchNormalization\n",
    "from tensorflow.python.keras.models import Model\n",
    "from tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "\n",
    "import argparse\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "from sklearn import preprocessing\n",
    "from google.cloud import storage\n",
    "\n",
    "import os\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "#import wfdb as wf\n",
    "from io import BytesIO\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from biosppy.signals import ecg\n",
    "import glob\n",
    "from scipy import signal\n",
    "import pandas as pd\n",
    "from tensorflow.python.keras.layers import Dense, Convolution1D, Convolution2D,MaxPool1D, Flatten, Dropout\n",
    "from tensorflow.python.keras.layers import Input\n",
    "from tensorflow.python.keras.models import Model\n",
    "from tensorflow.python.keras.layers.normalization import BatchNormalization\n",
    "import tensorflow.python.keras\n",
    "from tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from tensorflow.python.keras.utils.np_utils import to_categorical\n",
    "import tensorflow as tf\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from collections import Counter\n",
    "from sklearn import preprocessing\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os,glob\n",
    "import math\n",
    "import fnmatch\n",
    "import re\n",
    "import gc\n",
    "from google.cloud import storage\n",
    "import shutil\n",
    "from joblib import Parallel, delayed\n",
    "#import matplotlib.pyplot as plt\n",
    "import scipy.signal as scipysi\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn import metrics\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.multiclass import OneVsOneClassifier\n",
    "from sklearn.multiclass import OutputCodeClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.neighbors import NearestCentroid\n",
    "from sklearn.neighbors import RadiusNeighborsClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.linear_model import RidgeClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn import linear_model\n",
    "from sklearn.preprocessing import MinMaxScaler, MaxAbsScaler\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.layers import Bidirectional, GRU\n",
    "import shutil\n",
    "#import matplotlib as plt\n",
    "from tensorflow.python.keras.models import Model\n",
    "from tensorflow.python.keras.layers import Input, Dense, LSTM, multiply, concatenate, Activation, Masking, Reshape,CuDNNLSTM,GlobalMaxPooling1D, MaxPool2D,Flatten\n",
    "from tensorflow.python.keras.layers import Conv1D, Conv2D, BatchNormalization, GlobalAveragePooling1D, Permute, Dropout, GlobalAveragePooling2D,Concatenate\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "import numpy as np\n",
    "from imblearn.over_sampling import SMOTE, ADASYN, SVMSMOTE\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from tensorflow.keras import initializers\n",
    "from tensorflow.keras import regularizers, constraints\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from matplotlib.colors import ListedColormap\n",
    "\n",
    "from scipy import signal\n",
    "from scipy.fft import fftshift"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0316 23:10:48.938713 140735542186880 deprecation.py:506] From /anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0316 23:10:48.946288 140735542186880 deprecation.py:506] From /anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0316 23:10:48.953026 140735542186880 deprecation.py:506] From /anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Ones.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0316 23:10:49.034140 140735542186880 deprecation.py:506] From /anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "inputs_cnn (InputLayer)      [(None, 12, 300, 1)]      0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 12, 300, 32)       832       \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 12, 300, 32)       128       \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 12, 300, 32)       25632     \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 12, 300, 32)       128       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 6, 150, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 6, 150, 64)        51264     \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 6, 150, 64)        256       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 3, 75, 64)         0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 3, 75, 64)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 3, 75, 128)        204928    \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 3, 75, 128)        512       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 1, 37, 128)        0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 1, 37, 128)        0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 4736)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 512)               2425344   \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "main_output (Dense)          (None, 80746)             10416234  \n",
      "=================================================================\n",
      "Total params: 13,289,482\n",
      "Trainable params: 13,288,970\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "path='/Users/aring/IdeaProjects/ECG-biometric/src/100k-data/keras_export'\n",
    "loaded_model = tf.keras.experimental.load_from_saved_model(path)\n",
    "loaded_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['_TF_MODULE_IGNORED_PROPERTIES',\n",
       " '__call__',\n",
       " '__class__',\n",
       " '__delattr__',\n",
       " '__dict__',\n",
       " '__dir__',\n",
       " '__doc__',\n",
       " '__eq__',\n",
       " '__format__',\n",
       " '__ge__',\n",
       " '__getattribute__',\n",
       " '__gt__',\n",
       " '__hash__',\n",
       " '__init__',\n",
       " '__init_subclass__',\n",
       " '__le__',\n",
       " '__lt__',\n",
       " '__module__',\n",
       " '__ne__',\n",
       " '__new__',\n",
       " '__reduce__',\n",
       " '__reduce_ex__',\n",
       " '__repr__',\n",
       " '__setattr__',\n",
       " '__sizeof__',\n",
       " '__str__',\n",
       " '__subclasshook__',\n",
       " '__weakref__',\n",
       " '_activity_regularizer',\n",
       " '_add_inbound_node',\n",
       " '_add_unique_metric_name',\n",
       " '_add_variable_with_custom_getter',\n",
       " '_assert_compile_was_called',\n",
       " '_assert_weights_created',\n",
       " '_autocast',\n",
       " '_base_init',\n",
       " '_build_model_with_inputs',\n",
       " '_cache_output_metric_attributes',\n",
       " '_call_accepts_kwargs',\n",
       " '_call_arg_was_passed',\n",
       " '_call_fn_args',\n",
       " '_callable_losses',\n",
       " '_check_call_args',\n",
       " '_check_trainable_weights_consistency',\n",
       " '_checkpoint_dependencies',\n",
       " '_clear_losses',\n",
       " '_collect_input_masks',\n",
       " '_compile_distribution',\n",
       " '_compile_eagerly',\n",
       " '_compile_from_inputs',\n",
       " '_compile_time_distribution_strategy',\n",
       " '_compile_weights_loss_and_weighted_metrics',\n",
       " '_compute_dtype',\n",
       " '_compute_output_and_mask_jointly',\n",
       " '_deferred_dependencies',\n",
       " '_distribution_standardize_user_data',\n",
       " '_distribution_strategy',\n",
       " '_dtype',\n",
       " '_dtype_defaulted_to_floatx',\n",
       " '_dtype_policy',\n",
       " '_dynamic',\n",
       " '_eager_add_metric',\n",
       " '_eager_losses',\n",
       " '_expects_mask_arg',\n",
       " '_expects_training_arg',\n",
       " '_experimental_run_tf_function',\n",
       " '_feed_input_names',\n",
       " '_feed_input_shapes',\n",
       " '_feed_inputs',\n",
       " '_feed_loss_fns',\n",
       " '_feed_output_names',\n",
       " '_feed_output_shapes',\n",
       " '_feed_sample_weights',\n",
       " '_feed_targets',\n",
       " '_flatten',\n",
       " '_gather_children_attribute',\n",
       " '_gather_saveables_for_checkpoint',\n",
       " '_get_call_arg_value',\n",
       " '_get_callback_model',\n",
       " '_get_existing_metric',\n",
       " '_get_node_attribute_at_index',\n",
       " '_get_trainable_state',\n",
       " '_get_training_eval_metrics',\n",
       " '_graph',\n",
       " '_graph_network_add_loss',\n",
       " '_graph_network_add_metric',\n",
       " '_handle_activity_regularization',\n",
       " '_handle_deferred_dependencies',\n",
       " '_handle_metrics',\n",
       " '_handle_per_output_metrics',\n",
       " '_handle_weight_regularization',\n",
       " '_in_multi_worker_mode',\n",
       " '_inbound_nodes',\n",
       " '_init_call_fn_args',\n",
       " '_init_distributed_function_cache_if_not_compiled',\n",
       " '_init_graph_network',\n",
       " '_init_metric_attributes',\n",
       " '_init_set_name',\n",
       " '_init_subclassed_network',\n",
       " '_input_coordinates',\n",
       " '_input_layers',\n",
       " '_insert_layers',\n",
       " '_is_compiled',\n",
       " '_is_graph_network',\n",
       " '_is_layer',\n",
       " '_keras_api_names',\n",
       " '_keras_api_names_v1',\n",
       " '_layer_call_argspecs',\n",
       " '_layers',\n",
       " '_list_extra_dependencies_for_serialization',\n",
       " '_list_functions_for_serialization',\n",
       " '_lookup_dependency',\n",
       " '_loss_weights_list',\n",
       " '_losses',\n",
       " '_make_callback_model',\n",
       " '_make_execution_function',\n",
       " '_make_predict_function',\n",
       " '_make_test_function',\n",
       " '_make_train_function',\n",
       " '_maybe_build',\n",
       " '_maybe_cast_inputs',\n",
       " '_maybe_create_attribute',\n",
       " '_maybe_initialize_trackable',\n",
       " '_maybe_load_initial_epoch_from_ckpt',\n",
       " '_metrics',\n",
       " '_name',\n",
       " '_name_based_attribute_restore',\n",
       " '_name_based_restores',\n",
       " '_name_scope',\n",
       " '_nested_inputs',\n",
       " '_nested_outputs',\n",
       " '_network_nodes',\n",
       " '_no_dependency',\n",
       " '_nodes_by_depth',\n",
       " '_non_trainable_weights',\n",
       " '_obj_reference_counts',\n",
       " '_object_identifier',\n",
       " '_outbound_nodes',\n",
       " '_output_coordinates',\n",
       " '_output_layers',\n",
       " '_output_loss_metrics',\n",
       " '_output_mask_cache',\n",
       " '_output_shape_cache',\n",
       " '_output_tensor_cache',\n",
       " '_preload_simple_restoration',\n",
       " '_prepare_output_masks',\n",
       " '_prepare_sample_weights',\n",
       " '_prepare_skip_target_masks',\n",
       " '_prepare_total_loss',\n",
       " '_prepare_validation_data',\n",
       " '_process_target_tensor_for_compile',\n",
       " '_recompile_weights_loss_and_weighted_metrics',\n",
       " '_restore_from_checkpoint_position',\n",
       " '_reuse',\n",
       " '_run_eagerly',\n",
       " '_run_internal_graph',\n",
       " '_sample_weight_modes',\n",
       " '_scope',\n",
       " '_select_training_loop',\n",
       " '_self_name_based_restores',\n",
       " '_self_setattr_tracking',\n",
       " '_self_unconditional_checkpoint_dependencies',\n",
       " '_self_unconditional_deferred_dependencies',\n",
       " '_self_unconditional_dependency_names',\n",
       " '_self_update_uid',\n",
       " '_set_connectivity_metadata_',\n",
       " '_set_dtype_policy',\n",
       " '_set_input_attrs',\n",
       " '_set_inputs',\n",
       " '_set_mask_metadata',\n",
       " '_set_metric_attributes',\n",
       " '_set_optimizer',\n",
       " '_set_output_attrs',\n",
       " '_set_output_names',\n",
       " '_set_per_output_metric_attributes',\n",
       " '_set_trainable_state',\n",
       " '_setattr_tracking',\n",
       " '_should_compute_mask',\n",
       " '_single_restoration_from_checkpoint_position',\n",
       " '_standardize_user_data',\n",
       " '_symbolic_add_metric',\n",
       " '_symbolic_call',\n",
       " '_targets',\n",
       " '_tf_api_names',\n",
       " '_tf_api_names_v1',\n",
       " '_thread_local',\n",
       " '_track_layers',\n",
       " '_track_trackable',\n",
       " '_trackable_saver',\n",
       " '_tracking_metadata',\n",
       " '_trainable',\n",
       " '_trainable_weights',\n",
       " '_unconditional_checkpoint_dependencies',\n",
       " '_unconditional_dependency_names',\n",
       " '_unique_trainable_weights',\n",
       " '_update_sample_weight_modes',\n",
       " '_update_uid',\n",
       " '_updated_config',\n",
       " '_updates',\n",
       " '_validate_compile_param_for_distribution_strategy',\n",
       " '_validate_graph_inputs_and_outputs',\n",
       " '_validate_or_infer_batch_size',\n",
       " '_warn_about_input_casting',\n",
       " 'activity_regularizer',\n",
       " 'add_loss',\n",
       " 'add_metric',\n",
       " 'add_update',\n",
       " 'add_variable',\n",
       " 'add_weight',\n",
       " 'apply',\n",
       " 'build',\n",
       " 'built',\n",
       " 'call',\n",
       " 'compile',\n",
       " 'compute_mask',\n",
       " 'compute_output_shape',\n",
       " 'compute_output_signature',\n",
       " 'count_params',\n",
       " 'dtype',\n",
       " 'dynamic',\n",
       " 'evaluate',\n",
       " 'evaluate_generator',\n",
       " 'fit',\n",
       " 'fit_generator',\n",
       " 'from_config',\n",
       " 'get_config',\n",
       " 'get_input_at',\n",
       " 'get_input_mask_at',\n",
       " 'get_input_shape_at',\n",
       " 'get_layer',\n",
       " 'get_losses_for',\n",
       " 'get_output_at',\n",
       " 'get_output_mask_at',\n",
       " 'get_output_shape_at',\n",
       " 'get_updates_for',\n",
       " 'get_weights',\n",
       " 'inbound_nodes',\n",
       " 'input',\n",
       " 'input_mask',\n",
       " 'input_names',\n",
       " 'input_shape',\n",
       " 'input_spec',\n",
       " 'inputs',\n",
       " 'layers',\n",
       " 'load_weights',\n",
       " 'losses',\n",
       " 'metrics',\n",
       " 'metrics_names',\n",
       " 'name',\n",
       " 'name_scope',\n",
       " 'non_trainable_variables',\n",
       " 'non_trainable_weights',\n",
       " 'optimizer',\n",
       " 'outbound_nodes',\n",
       " 'output',\n",
       " 'output_mask',\n",
       " 'output_names',\n",
       " 'output_shape',\n",
       " 'outputs',\n",
       " 'predict',\n",
       " 'predict_generator',\n",
       " 'predict_on_batch',\n",
       " 'reset_metrics',\n",
       " 'reset_states',\n",
       " 'run_eagerly',\n",
       " 'sample_weights',\n",
       " 'save',\n",
       " 'save_weights',\n",
       " 'set_weights',\n",
       " 'state_updates',\n",
       " 'stateful',\n",
       " 'submodules',\n",
       " 'summary',\n",
       " 'supports_masking',\n",
       " 'test_on_batch',\n",
       " 'to_json',\n",
       " 'to_yaml',\n",
       " 'train_on_batch',\n",
       " 'trainable',\n",
       " 'trainable_variables',\n",
       " 'trainable_weights',\n",
       " 'updates',\n",
       " 'variables',\n",
       " 'weights',\n",
       " 'with_name_scope']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir(loaded_model)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### transfer learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "def network(num_of_people_in_data,learning_rate=0.01):\n",
    "    path='gs://ecg-data/ecg_iden_1data_chinaprivate_originalarchitecture_run2/keras_export'\n",
    "    model = tf.keras.experimental.load_from_saved_model(path)\n",
    "    model.summary()\n",
    "\n",
    "    # add new  layers\n",
    "    x = model.layers[-2].output\n",
    "    print('****1')\n",
    "\n",
    "   \n",
    "    x = Dense(50, activation='relu', name='transfer-dense')(x)\n",
    "    print('****2')\n",
    "\n",
    "    x = Dropout(0.5, name='transfer-dropout')(x) # Dropout layer to reduce overfitting\n",
    "    print('****3')\n",
    "    main_output = Dense(num_of_people_in_data, activation='softmax', name='transefer_output')(x)\n",
    "    print('****4')\n",
    "    model = Model(inputs=model.inputs, outputs=main_output)\n",
    "    print('****5')\n",
    "    for layer in model.layers[:-2]:\n",
    "        layer.trainable = False\n",
    "    for layer in model.layers[-2:]:\n",
    "        layer.trainable=True\n",
    "    # learning_rate=5e-5\n",
    "    print('****6')\n",
    "    model.compile(optimizer='adam', loss='sparse_categorical_crossentropy',metrics = ['sparse_categorical_accuracy'])\n",
    "    print('New model')\n",
    "    model.summary()\n",
    "\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0321 13:24:11.316303 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer\n",
      "W0321 13:24:11.337522 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.iter\n",
      "W0321 13:24:11.339313 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.beta_1\n",
      "W0321 13:24:11.341012 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.beta_2\n",
      "W0321 13:24:11.344110 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.decay\n",
      "W0321 13:24:11.345126 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer.learning_rate\n",
      "W0321 13:24:11.346091 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-0.kernel\n",
      "W0321 13:24:11.347779 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-0.bias\n",
      "W0321 13:24:11.350233 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-1.gamma\n",
      "W0321 13:24:11.351676 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-1.beta\n",
      "W0321 13:24:11.352911 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-2.kernel\n",
      "W0321 13:24:11.354099 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-2.bias\n",
      "W0321 13:24:11.354732 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-3.gamma\n",
      "W0321 13:24:11.356390 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-3.beta\n",
      "W0321 13:24:11.357810 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-4.kernel\n",
      "W0321 13:24:11.359078 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-4.bias\n",
      "W0321 13:24:11.360480 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-5.gamma\n",
      "W0321 13:24:11.362465 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-5.beta\n",
      "W0321 13:24:11.363710 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-6.kernel\n",
      "W0321 13:24:11.364948 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-6.bias\n",
      "W0321 13:24:11.367711 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-7.gamma\n",
      "W0321 13:24:11.369221 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-7.beta\n",
      "W0321 13:24:11.370405 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-8.kernel\n",
      "W0321 13:24:11.371610 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-8.bias\n",
      "W0321 13:24:11.372648 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-9.kernel\n",
      "W0321 13:24:11.373708 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-9.bias\n",
      "W0321 13:24:11.374839 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-10.kernel\n",
      "W0321 13:24:11.376461 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-10.bias\n",
      "W0321 13:24:11.378414 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-11.kernel\n",
      "W0321 13:24:11.379938 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-11.bias\n",
      "W0321 13:24:11.381120 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-0.kernel\n",
      "W0321 13:24:11.382799 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-0.bias\n",
      "W0321 13:24:11.384074 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-1.gamma\n",
      "W0321 13:24:11.385385 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-1.beta\n",
      "W0321 13:24:11.386354 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-2.kernel\n",
      "W0321 13:24:11.387781 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-2.bias\n",
      "W0321 13:24:11.389092 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-3.gamma\n",
      "W0321 13:24:11.390764 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-3.beta\n",
      "W0321 13:24:11.391706 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-4.kernel\n",
      "W0321 13:24:11.392922 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-4.bias\n",
      "W0321 13:24:11.395881 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-5.gamma\n",
      "W0321 13:24:11.397984 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-5.beta\n",
      "W0321 13:24:11.399451 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-6.kernel\n",
      "W0321 13:24:11.400731 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-6.bias\n",
      "W0321 13:24:11.404910 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-7.gamma\n",
      "W0321 13:24:11.406011 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-7.beta\n",
      "W0321 13:24:11.410305 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-8.kernel\n",
      "W0321 13:24:11.411701 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-8.bias\n",
      "W0321 13:24:11.412631 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-9.kernel\n",
      "W0321 13:24:11.413936 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-9.bias\n",
      "W0321 13:24:11.417327 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-10.kernel\n",
      "W0321 13:24:11.418590 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-10.bias\n",
      "W0321 13:24:11.420559 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-11.kernel\n",
      "W0321 13:24:11.422201 140735542186880 util.py:144] Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-11.bias\n",
      "W0321 13:24:11.432561 140735542186880 util.py:152] A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/alpha/guide/checkpoints#loading_mechanics for details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "inputs_cnn (InputLayer)      [(None, 12, 300, 1)]      0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 12, 300, 32)       832       \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 12, 300, 32)       128       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 6, 150, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 6, 150, 64)        51264     \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 6, 150, 64)        256       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 3, 75, 64)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 3, 75, 128)        204928    \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 3, 75, 128)        512       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 1, 37, 128)        0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 4736)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 128)               606336    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 50)                6450      \n",
      "_________________________________________________________________\n",
      "main_output (Dense)          (None, 38378)             1957278   \n",
      "=================================================================\n",
      "Total params: 2,827,984\n",
      "Trainable params: 2,827,536\n",
      "Non-trainable params: 448\n",
      "_________________________________________________________________\n",
      "****1\n",
      "****2\n",
      "****3\n",
      "****4\n",
      "****5\n",
      "****6\n",
      "New model\n",
      "Model: \"model_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "inputs_cnn (InputLayer)      [(None, 12, 300, 1)]      0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 12, 300, 32)       832       \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 12, 300, 32)       128       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 6, 150, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 6, 150, 64)        51264     \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 6, 150, 64)        256       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 3, 75, 64)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 3, 75, 128)        204928    \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 3, 75, 128)        512       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 1, 37, 128)        0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 4736)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 128)               606336    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 50)                6450      \n",
      "_________________________________________________________________\n",
      "transfer-dense (Dense)       (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "transfer-dropout (Dropout)   (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "transefer_output (Dense)     (None, 10)                510       \n",
      "=================================================================\n",
      "Total params: 873,766\n",
      "Trainable params: 510\n",
      "Non-trainable params: 873,256\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model=network(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### scoring\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "inputs_cnn (InputLayer)      [(None, 12, 300, 1)]      0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 12, 300, 32)       832       \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 12, 300, 32)       128       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 6, 150, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 6, 150, 64)        51264     \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 6, 150, 64)        256       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 3, 75, 64)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 3, 75, 128)        204928    \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 3, 75, 128)        512       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 1, 37, 128)        0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 4736)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 128)               606336    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 50)                6450      \n",
      "_________________________________________________________________\n",
      "main_output (Dense)          (None, 38378)             1957278   \n",
      "=================================================================\n",
      "Total params: 2,827,984\n",
      "Trainable params: 2,827,536\n",
      "Non-trainable params: 448\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "path='gs://ecg-data/ecg_iden_1data_chinaprivate_originalarchitecture_run2/keras_export'\n",
    "model = tf.keras.experimental.load_from_saved_model(path)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_np_array_from_gs_dirs(bucket_name,gs_dir_list):\n",
    "    client = storage.Client()\n",
    "    bucket = client.bucket(bucket_name)\n",
    "    files_processed=0\n",
    "    for gs_dir in gs_dir_list:\n",
    "        for blob in bucket.list_blobs(prefix=gs_dir):\n",
    "            if blob.name.endswith(\".csv\"):\n",
    "                files_processed=files_processed+1\n",
    "                file_full_path='gs://'+bucket_name+'/'+blob.name\n",
    "                print('processing: ', blob.name)\n",
    "                f = BytesIO(file_io.read_file_to_string(file_full_path, binary_mode=True))\n",
    "                np_datat_loaded = np.loadtxt(f, delimiter=',')\n",
    "                if files_processed==1:\n",
    "                    combined_data = np_datat_loaded\n",
    "                else:\n",
    "                    combined_data=np.concatenate((combined_data, np_datat_loaded), axis=0)\n",
    "\n",
    "    return combined_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_and_split(bucket,dir_list,scaler_type='MaxMin'):\n",
    "    all_data_loaded=load_np_array_from_gs_dirs(bucket,dir_list)\n",
    "    print('all data size cspchina: '+str(all_data_loaded.shape))\n",
    "    all_data_nparr=all_data_loaded[~np.any(np.isnan(all_data_loaded) , axis=1)]\n",
    "    all_data_nparr=all_data_nparr.reshape((all_data_nparr.shape[0], 12, 305))\n",
    "    X=all_data_nparr[:,:,:300]\n",
    "    X=X.reshape(X.shape[0],12*300)\n",
    "    if scaler_type=='Standard':\n",
    "        scaler = StandardScaler()\n",
    "        scaler.fit(X)\n",
    "        X=scaler.transform(X)\n",
    "    elif scaler_type=='MaxMin':\n",
    "        x_min = X.min()\n",
    "        x_max = X.max()\n",
    "        print('max min seen in data cspchina: '+str((x_min,x_max)))\n",
    "        X = (X - x_min)/(x_max-x_min)\n",
    "    elif scaler_type=='None':\n",
    "        pass\n",
    "    \n",
    "    else:\n",
    "        print('Error: Wrong scaler type!')\n",
    "\n",
    "    y=all_data_nparr[:,0,300]\n",
    "    X=X.reshape((-1, 12, 300, 1))\n",
    "\n",
    "    return X,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing:  100k-data/china_private1/output-small/output-00002-of-00055.csv\n",
      "all data size cspchina: (687, 3660)\n"
     ]
    }
   ],
   "source": [
    "X,y=load_and_split('ecg-data',['100k-data/china_private1/output-small' ],scaler_type='None')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "train_inds, test_inds = next(StratifiedShuffleSplit(n_splits=2, test_size=0.2, random_state=52).split(X,y))\n",
    "trainX, testX = X[train_inds], X[test_inds]\n",
    "trainY, testY = y[train_inds], y[test_inds]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainX.shapehttp://localhost:8888/notebooks/ECG/load-google-ai-model.ipynb#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(np.concatenate((trainY,testY)))\n",
    "train_y_encoded = le.transform(trainY)\n",
    "test_y_encoded = le.transform(testY)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[28237.0, 28178.0, 28234.0, 28222.0, 28253.0, 28060.0, 28197.0, 28145.0, 28114.0, 28229.0, 28161.0, 28253.0, 28130.0, 28068.0, 28189.0, 28263.0, 28155.0, 28080.0, 28223.0, 28242.0, 28245.0, 28224.0, 28110.0, 28156.0, 28166.0, 28253.0, 28102.0, 28251.0, 28078.0, 28128.0, 28068.0, 28144.0, 28132.0, 28145.0, 28262.0, 28120.0, 28204.0, 28127.0]\n",
      "[27393, 27337, 27390, 27378, 27407, 27223, 27354, 27305, 27275, 27385, 27320, 27407, 27291, 27231, 27347, 27417, 27314, 27242, 27379, 27397, 27400, 27380, 27271, 27315, 27325, 27407, 27263, 27405, 27240, 27289, 27231, 27304, 27293, 27305, 27416, 27281, 27361, 27288]\n"
     ]
    }
   ],
   "source": [
    "print(list(testY[100:200]))\n",
    "\n",
    "y_pred=model.predict(testX[100:200,:,:,:])\n",
    "print(list(y_pred.argmax(axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.31160049e+03,  1.23653643e+03,  1.05850645e+03,  8.31470512e+02,\n",
       "        5.97524872e+02,  3.85707528e+02,  2.22998788e+02,  1.20444285e+02,\n",
       "        7.20129407e+01,  5.45662049e+01,  3.97540966e+01,  2.44909383e+01,\n",
       "        1.17892945e+01, -9.66595212e-01, -6.56882635e+00, -1.12248779e+01,\n",
       "       -1.46430010e+01, -1.77213941e+01, -2.03745520e+01, -2.28352290e+01,\n",
       "       -2.42738120e+01, -2.47831595e+01, -2.47294377e+01, -2.49456478e+01,\n",
       "       -2.54042734e+01, -2.56311979e+01, -2.49926803e+01, -2.37472067e+01,\n",
       "       -2.24341815e+01, -2.13131188e+01, -2.00290600e+01, -1.82762569e+01,\n",
       "       -1.62039275e+01, -1.43865369e+01, -1.32877179e+01, -1.29293462e+01,\n",
       "       -1.28909330e+01, -1.26557528e+01, -1.20451374e+01, -1.11710645e+01,\n",
       "       -1.01599802e+01, -8.90286388e+00, -7.42131171e+00, -5.87906207e+00,\n",
       "       -4.43680581e+00, -3.11440391e+00, -1.99894961e+00, -1.04472397e+00,\n",
       "       -1.25338390e-01,  1.01926799e+00,  2.49987375e+00,  4.44918671e+00,\n",
       "        7.04085656e+00,  1.04369775e+01,  1.44538449e+01,  1.89173558e+01,\n",
       "        2.38991098e+01,  3.00160791e+01,  3.77796461e+01,  4.68936877e+01,\n",
       "        5.63162641e+01,  6.54682636e+01,  7.44378289e+01,  8.39973375e+01,\n",
       "        9.50952019e+01,  1.07785027e+02,  1.20654492e+02,  1.32169928e+02,\n",
       "        1.41768435e+02,  1.50318761e+02,  1.60064146e+02,  1.72426449e+02,\n",
       "        1.85426895e+02,  1.97249924e+02,  2.07623208e+02,  2.17586389e+02,\n",
       "        2.27880688e+02,  2.38347228e+02,  2.48611551e+02,  2.59086269e+02,\n",
       "        2.70786692e+02,  2.83664038e+02,  2.95812137e+02,  3.06108280e+02,\n",
       "        3.14278021e+02,  3.20465243e+02,  3.25307393e+02,  3.28890358e+02,\n",
       "        3.30301543e+02,  3.29072334e+02,  3.25452946e+02,  3.20000025e+02,\n",
       "        3.11673175e+02,  2.97226560e+02,  2.77735806e+02,  2.57283651e+02,\n",
       "        2.36153286e+02,  2.14041635e+02,  1.92436550e+02,  1.71815417e+02,\n",
       "        1.51656373e+02,  1.31832248e+02,  1.13081040e+02,  9.55263094e+01,\n",
       "        7.71983980e+01,  5.95964429e+01,  4.41968370e+01,  3.02197106e+01,\n",
       "        1.64204280e+01,  6.01379803e+00,  7.26082303e-01, -2.04634093e+00,\n",
       "       -3.57467123e+00, -4.66844370e+00, -5.81145393e+00, -7.08436755e+00,\n",
       "       -8.11016405e+00, -8.54645380e+00, -8.32409856e+00, -7.83759460e+00,\n",
       "       -7.60746082e+00, -7.87496263e+00, -8.39558254e+00, -8.73233862e+00,\n",
       "       -8.60547114e+00, -8.06373484e+00, -7.39731395e+00, -6.94078341e+00,\n",
       "       -6.75926346e+00, -6.64679618e+00, -6.47305896e+00, -6.28525657e+00,\n",
       "       -6.06231342e+00, -5.62454015e+00, -4.73678430e+00, -3.32196293e+00,\n",
       "       -1.63062812e+00, -1.86870953e-01,  6.42324767e-01,  1.00545511e+00,\n",
       "        1.34183907e+00,  1.78631907e+00,  2.03644209e+00,  1.74665800e+00,\n",
       "        1.01138838e+00,  3.62068754e-01,  2.34139659e-01,  4.82859299e-01,\n",
       "        5.52228601e-01,  3.25486179e-02, -9.24000878e-01, -1.82972868e+00,\n",
       "       -2.26562472e+00, -2.09380056e+00, -1.34834635e+00, -2.23228026e-01,\n",
       "        9.38754849e-01,  1.79745524e+00,  2.33792945e+00,  2.66795385e+00,\n",
       "        2.73635714e+00,  2.35213368e+00,  1.45704507e+00,  1.86797309e-01,\n",
       "       -1.15024254e+00, -2.37996791e+00, -3.56539734e+00, -4.84943270e+00,\n",
       "       -6.03718953e+00, -6.72922918e+00, -6.74837265e+00, -6.42325649e+00,\n",
       "       -6.10384814e+00, -5.86168718e+00, -5.51698372e+00, -4.99235294e+00,\n",
       "       -4.38326172e+00, -3.97807817e+00, -3.92904163e+00, -4.20514375e+00,\n",
       "       -4.57394704e+00, -4.78124714e+00, -4.50968939e+00, -3.73889755e+00,\n",
       "       -2.87075297e+00, -2.61176978e+00, -3.20966622e+00, -4.24504408e+00,\n",
       "       -5.00109561e+00, -5.28338232e+00, -5.45343102e+00, -6.18787019e+00,\n",
       "       -7.72765880e+00, -9.66139863e+00, -1.12066405e+01, -1.20981185e+01,\n",
       "       -1.25345901e+01, -1.30032220e+01, -1.38056505e+01, -1.49568958e+01,\n",
       "       -1.59699843e+01, -1.63520559e+01, -1.59433339e+01, -1.52305509e+01,\n",
       "       -1.46624341e+01, -1.42755373e+01, -1.35363247e+01, -1.20825025e+01,\n",
       "       -1.01133567e+01, -8.57373565e+00, -8.07594622e+00, -8.42790077e+00,\n",
       "       -8.78196041e+00, -8.71404118e+00, -8.20473515e+00, -7.62853966e+00,\n",
       "       -7.23212338e+00, -7.12550415e+00, -6.97068508e+00, -6.58315000e+00,\n",
       "       -5.96658135e+00, -5.52002327e+00, -5.37611492e+00, -5.52313616e+00,\n",
       "       -5.55451168e+00, -5.22949027e+00, -4.50512023e+00, -3.94577412e+00,\n",
       "       -3.89944507e+00, -4.40342241e+00, -4.91357864e+00, -5.23093669e+00,\n",
       "       -5.44710016e+00, -6.07974723e+00, -7.19826776e+00, -8.64803785e+00,\n",
       "       -9.77280255e+00, -1.02548056e+01, -9.94069287e+00, -9.28428320e+00,\n",
       "       -8.47864391e+00, -7.56098743e+00, -6.00425958e+00, -3.69660653e+00,\n",
       "       -4.72503009e-01,  3.52539373e+00,  8.48333681e+00,  1.42434920e+01,\n",
       "        2.09958567e+01,  2.85421411e+01,  3.65965027e+01,  4.41356723e+01,\n",
       "        5.10893143e+01,  5.77807347e+01,  6.49177393e+01,  7.24341060e+01,\n",
       "        8.04755174e+01,  8.91378413e+01,  9.93668597e+01,  1.10596002e+02,\n",
       "        1.21700202e+02,  1.31995224e+02,  1.43464507e+02,  1.59088009e+02,\n",
       "        1.73881365e+02,  1.77219074e+02,  1.65700571e+02,  1.38798933e+02,\n",
       "        1.09794085e+02,  8.51824782e+01,  6.26935675e+01,  4.27924034e+01,\n",
       "        2.64354695e+01,  1.17785446e+01,  1.08652730e+00, -4.52681575e+00,\n",
       "       -7.72927245e+00, -1.04742987e+01, -1.16945630e+01, -1.21844571e+01,\n",
       "       -1.13347109e+01, -1.07867871e+01, -9.27784110e+00, -8.15564604e+00,\n",
       "       -5.23062780e+00, -2.97166293e+00,  1.24707466e+00,  2.61342248e+00,\n",
       "        1.14094899e+01,  1.11510061e+01, -5.27814716e+00, -4.67761751e+00,\n",
       "        3.01091818e+00,  4.28673205e+01,  1.29872048e+02,  2.72454259e+02,\n",
       "        4.82991263e+02,  7.54635405e+02,  1.03128051e+03,  1.24156788e+03])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testX[0,1,:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28257.0"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testY[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
